﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using AngleSharp;
using System.Text.RegularExpressions;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {
            for (int i = 0; i < 500; i++)
            {
                collect(i);

            }


            Console.WriteLine("complete");
            Console.Read();
        }
        static void collect(int page)
        {
            string url = "http://jandan.net/page/" + page;
            Console.WriteLine(url);

            var r = HttpGet(url, "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36782"+Guid.NewGuid().ToString());

            var document = DocumentBuilder.Html(r);
            var ahrefs = document.QuerySelectorAll("#content .column .title2 a");
            foreach (var item in ahrefs)
            {

                var href = item.GetAttribute("href");

                collectDec(href);



            }
        }
        static void collectDec(string url)
        {
            Console.WriteLine(url);
            var body = HttpGet(url, "Mozilla / 5.0(iPhone; U; CPU iPhone OS 4_2_1 like Mac OS X; en - us) AppleWebKit / 533.17.9(KHTML, like Gecko) Version / 5.0.2 Mobile / 8C148 Safari/ 6533.18.5" + Guid.NewGuid().ToString());
            var document = DocumentBuilder.Html(body);
            var entry = document.QuerySelector("#maincontent .post .entry");
            entry.QuerySelector(".jandan-zan").Remove();
            entry.QuerySelector("p:last-child").Remove();
            var contents = entry.InnerHtml;

        }

        public static string HttpGet(string Url, string UserAgent)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
            request.Method = "GET";
            request.ContentType = "text/html;charset=UTF-8";
            request.UserAgent = UserAgent;
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            Stream myResponseStream = response.GetResponseStream();
            StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
            string retString = myStreamReader.ReadToEnd();
            myStreamReader.Close();
            myResponseStream.Close();

            return retString;
        }


    }
}
